# -*- coding: UTF-8 -*-
# Spider for 'asos.com'

from urlparse import urljoin

from scrapy.spider import BaseSpider
from scrapy.http import  Request
from scrapy.selector import HtmlXPathSelector
from littlecrawler.items import ScrapedItem

import sys

class Asos(BaseSpider):
    
    # To run this scrapy crawl asos --set FEED_URI=/Sitemaps/asos.json --set FEED_FORMAT=json

    name = 'asos'
    start_urls = ['http://www.asos.com/Woman/',
                  'http://www.asos.com/man/']
    URL = 'http://www.asos.com'
    visited = set()
    
    def parse(self, response):
        page = HtmlXPathSelector(response)
        for cat in page.select("//div[@id='side-nav']//a/@href").extract():
            yield Request(self.URL+cat, callback=self.parse, encoding = 'latin-1')
        else:
            for i in self.list_items(response):
                yield i
    
    
    
    def list_items(self, response):
        page = HtmlXPathSelector(response)

        for url in page.select("//ul[@id='items']//a[@class='desc']/@href").extract():
            if url not in self.visited:
                self.visited.add(url)

                yield Request(urljoin(response.url, url), callback=self.extract_data, encoding = 'latin-1')              

        next_page = page.select("//li[@class='page-skip']/a[contains(text(), 'Next')]/@href").extract()
        if next_page:
            yield Request(urljoin(response.url, next_page[0]), callback=self.list_items, encoding = 'latin-1')
            
    def extract_data(self, response):
        page = HtmlXPathSelector(response)
        sex = page.select('//meta[@name="DCSext.pg_n"]/@content').extract()
        name = page.select('//span[@class="product_title"]/text()').extract()
        price =  page.select('//div[@class="product_price"]//span//text()').extract()
        image1 =  page.select('//div[@class="image1"]/img/@src').extract()
        image2 =  page.select('//div[@class="image2"]/img/@src').extract()
        image3 =  page.select('//div[@class="image3"]/img/@src').extract()
        image4 =  page.select('//div[@class="image4"]/img/@src').extract()
        description =  page.select('//div[@class="product-description"]/ul/li/text()').extract()
        designer =  page.select('//meta[@name="DCSext.brand"]/@content').extract()
        type =  page.select('//meta[@name="DCSext.pg_s"]/@content').extract()
        facebookURL = page.select('//meta[@property="og:url"]/@content').extract()

        
        item = ScrapedItem()
        item['source'] = 'asos';
        item['name'] = name;
        item['sex'] = sex;
        item['price'] = price;
        item['image1'] = image1;
        item['image2'] = image2;
        item['image3'] = image3;
        item['image4'] = image4;
        item['description'] = description;
        item['designer'] = designer;
        item['type'] = type;
        item['facebookURL'] = facebookURL;
        item['url'] = response.url;
        item['category1'] = '';
        yield item
        
        
